<?php

set_time_limit(0);
//db
$link_id = mysql_pconnect('127.0.0.1', 'root', 'root') or die('db connect fail.');
mysql_select_db('juzi_searcher', $link_id);                                   
mysql_query('set charset utf8'); 

//init
$processed_urls = array();
$unprocessed_urls = array();
$entry_num = get_entry_num();
$page_size = 100;
$page_no = 1;

//logic
do{
	$start_id = ($page_no-1)*$page_size;
	if($start_id > $entry_num){
		break;
	}
	$sql = "SELECT * FROM blog_entry LIMIT {$start_id},{$page_size}";
	$result = mysql_query($sql, $link_id);
	while($entry = mysql_fetch_assoc($result)){
		blog_crawl($entry);
	}
	$page_no++;
}while(true);

//function list
//获取链接内容
function get_contents($uri){
	
}
//抓取
function blog_crawl($entry){
	update_entry_log($entry);
	//广度优先算法抓取
}
//更新$entry下的日志内容，由于博客更新频率慢，暂时不实现
function update_entry_log($entry){
	return true;
}
//判断$url是否已经抓取
function is_processed($url){
	return false;
}
//获取独立博客入口总数
function get_entry_num(){
        global $link_id;
        $sql = 'SELECT count(*) as num FROM blog_entry;';
        $result = mysql_query($sql, $link_id);
        return mysql_result($result, 0, 0);
}
/** 日志内容入库 @fansichi
 *  $data 日志内容
    $data[entry_id]  抓取网站ID
	$data[title]     网站标题
	$data['content'] 网站内容
	$data['url']     log的链接来源	
 */

function add_log($data){
	global $link_id;	
	if(!is_array($data)) 
		return false;
	$data = array_map("mysql_real_escape_string", $data );	
	$now = time();
	$sql = "INSERT INTO blog_log(entry_id,title,content,url,update_time) VALUES('{$data['entry_id']}','{$data['title']}','{$data['content']}','{$data['url']}',{$now})";	
	return mysql_query($sql, $link_id);
}
//是否属于某个入口 @fansichi
function is_reside_entry($url, $entry){
	if( !is_array($entry) )  
		return false;
	//博客站点url
	$entry_site = strtolower($entry['url']);
	
	$match_part = parse_url($url);
	$url_host = $match_part['host'];
	if($url_host == '') 
		return false;
	
	$arr_host =  explode(".", $url_host);
	//不规则的host，丢弃处理
	if( count($arr_host) != 3)
		return false;
		
	//得到url的域名,格式为如： juzisource.com		
	$url_domain = implode(".", array_slice($arr_host,1));	
	//找到，认为是该网站内部的链接
	if( strpos($entry_site, $url_domain) !== false ) 
		return true;	
	return false;
}
//相对路径转绝对路径 @fansichi
function get_absolute_path($refer_url, $relative_path ){	
	//refer_url必须是绝对路径，否则直接返回false
	if( !is_absolute_path($refer_url)) 
		return false;
	
	//relative_path如果是绝对路径，不做任何处理直接返回
	if(is_absolute_path($relative_path)) 
		return $relative_path;
		
	$absolute_path = '';	
	$match_part = parse_url($refer_url);
	
	//分析当前的目录结构
	$match_part['path'] = preg_replace("|/+|", "/", $match_part['path']);
	$refer_dir_levels =  explode("/", $match_part['path']);	

	//剔除最后一个元素，即最终页面
	array_pop($refer_dir_levels);
	//当前的层级数量
	$level_num = count($refer_dir_levels);
	
	$relative_path = preg_replace("|/+|", "/", $relative_path);
	$relative_dir_levels = explode("/", $relative_path);	
	$termial_page = array_pop($relative_dir_levels);
	
	if(count($relative_dir_levels) > 0 ) {
		foreach($relative_dir_levels as $relative_dir_level) {
			//层级数减一,说明目录往上翻了一级
			if($relative_dir_level == '..') {
				$level_num -- ;
			}
		}
	}
	
	$absolute_path .= $match_part['scheme'] . "://" . $match_part['host'] ;
	if( isset($match_part['port']) && !empty($match_part['port'])) {
		$absolute_path .= ":" . $match_part['port'] ;
	}
	if( $level_num > 0) {
		for($i = 0; $i < $level_num ; $i++) {
			if($refer_dir_levels[$i] != '') {
				$absolute_path .= "/" . $refer_dir_levels[$i];
			}		
		}		
	}
	$absolute_path .= "/" . $termial_page;
	return $absolute_path;
}
//判断url是否绝对路径  @fansichi
function is_absolute_path($relative_path) {	
	$match_part = parse_url($relative_path);
	if( isset($match_part['scheme']) && $match_part['scheme'] != '' &&
		isset($match_part['host']) && $match_part['host'] != '') {
		return true;
	}	
	return false;
}
//日志内容简单过滤(取body内容，去css、js、html标签) @fansichi
function log_filter($str){
	$search = array(   "'<script[^>]*?>.*?</script>'si",	// strip out javascript	
						"'<style[^>]*?>.*?</style>'si",	    // strip out css
						"'<title[^>]*?>.*?</title>'",       // strip title
						"'<[\/\!]*?[^<>]*?>'si",			// strip out html tags
						"'([\r\n])[\s]+'",					// strip out white space
						"'&(quot|#34|#034|#x22);'i",		// replace html entities
						"'&(amp|#38|#038|#x26);'i",			// added hexadecimal values
						"'&(lt|#60|#060|#x3c);'i",
						"'&(gt|#62|#062|#x3e);'i",
						"'&(nbsp|#160|#xa0);'i",
						"'&(iexcl|#161);'i",
						"'&(cent|#162);'i",
						"'&(pound|#163);'i",
						"'&(copy|#169);'i",
						"'&(reg|#174);'i",
						"'&(deg|#176);'i",
						"'&(#39|#039|#x27);'",
						);
	$replace = array(	"",
						"",
						"",
						"",
						"\\1",
						"\"",
						"&",
						"<",
						">",
						" ",
						chr(161),
						chr(162),
						chr(163),
						chr(169),
						chr(174),
						chr(176),
						chr(39),
					);
				
	$text = preg_replace($search,$replace,$str);
							
	return $text;
	
}
?>
